{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"><li><span><a href=\"#Exploring-OpenAI\" data-toc-modified-id=\"Exploring-OpenAI-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Exploring OpenAI</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reinforcement Learning <a class=\"tocSkip\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NumPy:1.13.1\n",
      "TensorFlow:1.4.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras:2.0.9\n",
      "OpenAI Gym: 0.9.4\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "print(\"NumPy:{}\".format(np.__version__))\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(123)\n",
    "print(\"TensorFlow:{}\".format(tf.__version__))\n",
    "\n",
    "import keras\n",
    "print(\"Keras:{}\".format(keras.__version__))\n",
    "\n",
    "import gym\n",
    "print('OpenAI Gym:',gym.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OpenAI 101"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Environments in Gym version 0.9.4 : 777\n"
     ]
    }
   ],
   "source": [
    "# print how many environments available\n",
    "\n",
    "all_env = list(gym.envs.registry.all())\n",
    "\n",
    "print('Total Environments in Gym version {} : {}'.format(gym.__version__,len(all_env)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EnvSpec(Assault-ramDeterministic-v0)\n",
      "EnvSpec(AirRaidNoFrameskip-v4)\n",
      "EnvSpec(UpNDown-ram-v4)\n",
      "EnvSpec(Robotank-ramNoFrameskip-v4)\n",
      "EnvSpec(Kangaroo-ram-v0)\n",
      "EnvSpec(Kangaroo-ramNoFrameskip-v0)\n",
      "EnvSpec(CrazyClimber-ramNoFrameskip-v4)\n",
      "EnvSpec(SemisuperPendulumNoise-v0)\n",
      "EnvSpec(MsPacman-ramNoFrameskip-v4)\n",
      "EnvSpec(CrazyClimber-ram-v0)\n",
      "EnvSpec(InvertedDoublePendulum-v1)\n",
      "EnvSpec(VentureNoFrameskip-v4)\n",
      "EnvSpec(ChopperCommand-ramDeterministic-v0)\n",
      "EnvSpec(OffSwitchCartpoleProb-v0)\n",
      "EnvSpec(AssaultDeterministic-v0)\n",
      "EnvSpec(Freeway-ram-v0)\n",
      "EnvSpec(BoxingDeterministic-v4)\n",
      "EnvSpec(Venture-ramNoFrameskip-v0)\n",
      "EnvSpec(Hero-ramNoFrameskip-v0)\n",
      "EnvSpec(Assault-v0)\n",
      "EnvSpec(WizardOfWorNoFrameskip-v4)\n",
      "EnvSpec(Freeway-ramDeterministic-v4)\n",
      "EnvSpec(Berzerk-ram-v4)\n",
      "EnvSpec(Boxing-ram-v0)\n",
      "EnvSpec(RoadRunner-ramDeterministic-v0)\n",
      "EnvSpec(Frostbite-ram-v4)\n",
      "EnvSpec(AlienNoFrameskip-v4)\n",
      "EnvSpec(Bowling-ram-v4)\n",
      "EnvSpec(Breakout-ramDeterministic-v4)\n",
      "EnvSpec(BeamRider-ramDeterministic-v4)\n",
      "EnvSpec(Zaxxon-ramDeterministic-v0)\n",
      "EnvSpec(ElevatorAction-v4)\n",
      "EnvSpec(Jamesbond-ramNoFrameskip-v0)\n",
      "EnvSpec(AirRaid-ram-v4)\n",
      "EnvSpec(Amidar-v0)\n",
      "EnvSpec(PongNoFrameskip-v4)\n",
      "EnvSpec(Tutankham-ram-v4)\n",
      "EnvSpec(Pooyan-v4)\n",
      "EnvSpec(Gravitar-v4)\n",
      "EnvSpec(CentipedeNoFrameskip-v4)\n",
      "EnvSpec(TennisNoFrameskip-v4)\n",
      "EnvSpec(RoadRunnerNoFrameskip-v0)\n",
      "EnvSpec(Robotank-ramNoFrameskip-v0)\n",
      "EnvSpec(FishingDerby-ram-v0)\n",
      "EnvSpec(Striker-v0)\n",
      "EnvSpec(VideoPinball-ramDeterministic-v0)\n",
      "EnvSpec(KungFuMaster-v0)\n",
      "EnvSpec(WizardOfWorNoFrameskip-v0)\n",
      "EnvSpec(Breakout-ramNoFrameskip-v0)\n",
      "EnvSpec(NameThisGameNoFrameskip-v4)\n",
      "EnvSpec(UpNDown-v4)\n",
      "EnvSpec(PitfallNoFrameskip-v0)\n",
      "EnvSpec(Pong-v0)\n",
      "EnvSpec(FishingDerby-ramDeterministic-v0)\n",
      "EnvSpec(DemonAttackDeterministic-v0)\n",
      "EnvSpec(Solaris-ram-v4)\n",
      "EnvSpec(Assault-ramNoFrameskip-v0)\n",
      "EnvSpec(BeamRider-ramNoFrameskip-v4)\n",
      "EnvSpec(StarGunner-ram-v0)\n",
      "EnvSpec(Thrower-v0)\n",
      "EnvSpec(TimePilotNoFrameskip-v0)\n",
      "EnvSpec(HalfCheetah-v1)\n",
      "EnvSpec(QbertDeterministic-v4)\n",
      "EnvSpec(AsteroidsNoFrameskip-v0)\n",
      "EnvSpec(Phoenix-v4)\n",
      "EnvSpec(SpaceInvaders-v0)\n",
      "EnvSpec(BattleZone-ram-v0)\n",
      "EnvSpec(Jamesbond-ram-v0)\n",
      "EnvSpec(RiverraidDeterministic-v4)\n",
      "EnvSpec(RiverraidDeterministic-v0)\n",
      "EnvSpec(ElevatorAction-ram-v0)\n",
      "EnvSpec(JourneyEscape-v0)\n",
      "EnvSpec(PrivateEye-ramNoFrameskip-v4)\n",
      "EnvSpec(Carnival-v0)\n",
      "EnvSpec(Berzerk-ramNoFrameskip-v4)\n",
      "EnvSpec(SkiingNoFrameskip-v4)\n",
      "EnvSpec(FrozenLake8x8-v0)\n",
      "EnvSpec(Hero-ramDeterministic-v0)\n",
      "EnvSpec(Centipede-ramDeterministic-v0)\n",
      "EnvSpec(KrullNoFrameskip-v4)\n",
      "EnvSpec(IceHockey-ram-v4)\n",
      "EnvSpec(Riverraid-ramNoFrameskip-v0)\n",
      "EnvSpec(Freeway-ramNoFrameskip-v4)\n",
      "EnvSpec(PredictObsCartpole-v0)\n",
      "EnvSpec(IceHockey-ram-v0)\n",
      "EnvSpec(ChopperCommand-ramNoFrameskip-v0)\n",
      "EnvSpec(BerzerkDeterministic-v0)\n",
      "EnvSpec(AmidarNoFrameskip-v0)\n",
      "EnvSpec(Phoenix-v0)\n",
      "EnvSpec(Tennis-ramDeterministic-v4)\n",
      "EnvSpec(Tennis-ramNoFrameskip-v0)\n",
      "EnvSpec(MsPacman-v0)\n",
      "EnvSpec(RoadRunner-v4)\n",
      "EnvSpec(YarsRevengeNoFrameskip-v0)\n",
      "EnvSpec(Solaris-ramNoFrameskip-v0)\n",
      "EnvSpec(PitfallDeterministic-v0)\n",
      "EnvSpec(Breakout-ram-v4)\n",
      "EnvSpec(Assault-ram-v0)\n",
      "EnvSpec(Carnival-ram-v0)\n",
      "EnvSpec(Tennis-v0)\n",
      "EnvSpec(Krull-v0)\n",
      "EnvSpec(AtlantisNoFrameskip-v0)\n",
      "EnvSpec(TimePilot-ramNoFrameskip-v0)\n",
      "EnvSpec(AmidarDeterministic-v4)\n",
      "EnvSpec(JourneyEscapeDeterministic-v4)\n",
      "EnvSpec(Gopher-ramDeterministic-v4)\n",
      "EnvSpec(Pong-ramDeterministic-v4)\n",
      "EnvSpec(ZaxxonNoFrameskip-v4)\n",
      "EnvSpec(IceHockeyDeterministic-v4)\n",
      "EnvSpec(DemonAttack-ramDeterministic-v0)\n",
      "EnvSpec(Gopher-ramNoFrameskip-v4)\n",
      "EnvSpec(Roulette-v0)\n",
      "EnvSpec(Pong-ramNoFrameskip-v0)\n",
      "EnvSpec(BeamRider-v4)\n",
      "EnvSpec(NameThisGame-v4)\n",
      "EnvSpec(BeamRider-v0)\n",
      "EnvSpec(JourneyEscape-ramNoFrameskip-v4)\n",
      "EnvSpec(TimePilotNoFrameskip-v4)\n",
      "EnvSpec(Amidar-ram-v4)\n",
      "EnvSpec(BeamRiderDeterministic-v4)\n",
      "EnvSpec(Hero-v4)\n",
      "EnvSpec(Riverraid-v4)\n",
      "EnvSpec(SkiingDeterministic-v0)\n",
      "EnvSpec(HumanoidStandup-v1)\n",
      "EnvSpec(Carnival-ramDeterministic-v4)\n",
      "EnvSpec(Breakout-v4)\n",
      "EnvSpec(SkiingNoFrameskip-v0)\n",
      "EnvSpec(EnduroDeterministic-v4)\n",
      "EnvSpec(BowlingDeterministic-v0)\n",
      "EnvSpec(GopherDeterministic-v0)\n",
      "EnvSpec(TwoRoundDeterministicReward-v0)\n",
      "EnvSpec(CrazyClimberDeterministic-v0)\n",
      "EnvSpec(Asteroids-v4)\n",
      "EnvSpec(NameThisGameDeterministic-v0)\n",
      "EnvSpec(JourneyEscape-ramDeterministic-v4)\n",
      "EnvSpec(DemonAttack-ram-v4)\n",
      "EnvSpec(BoxingNoFrameskip-v4)\n",
      "EnvSpec(VideoPinball-ramNoFrameskip-v4)\n",
      "EnvSpec(NameThisGame-ramDeterministic-v0)\n",
      "EnvSpec(Taxi-v2)\n",
      "EnvSpec(PhoenixNoFrameskip-v0)\n",
      "EnvSpec(Pooyan-v0)\n",
      "EnvSpec(DoubleDunk-ramNoFrameskip-v4)\n",
      "EnvSpec(ElevatorActionNoFrameskip-v4)\n",
      "EnvSpec(ConvergenceControl-v0)\n",
      "EnvSpec(SpaceInvaders-ramDeterministic-v0)\n",
      "EnvSpec(Enduro-ram-v0)\n",
      "EnvSpec(Bowling-ramNoFrameskip-v4)\n",
      "EnvSpec(RoadRunner-ramDeterministic-v4)\n",
      "EnvSpec(AirRaid-ram-v0)\n",
      "EnvSpec(CarRacing-v0)\n",
      "EnvSpec(JourneyEscapeDeterministic-v0)\n",
      "EnvSpec(Humanoid-v1)\n",
      "EnvSpec(VideoPinballNoFrameskip-v4)\n",
      "EnvSpec(DemonAttack-ram-v0)\n",
      "EnvSpec(IceHockey-ramNoFrameskip-v4)\n",
      "EnvSpec(JamesbondDeterministic-v0)\n",
      "EnvSpec(BattleZone-ramNoFrameskip-v4)\n",
      "EnvSpec(Boxing-ramDeterministic-v4)\n",
      "EnvSpec(BattleZone-ramDeterministic-v0)\n",
      "EnvSpec(ZaxxonNoFrameskip-v0)\n",
      "EnvSpec(CentipedeDeterministic-v0)\n",
      "EnvSpec(NameThisGame-ramNoFrameskip-v0)\n",
      "EnvSpec(Phoenix-ramNoFrameskip-v0)\n",
      "EnvSpec(BreakoutDeterministic-v4)\n",
      "EnvSpec(DemonAttack-v4)\n",
      "EnvSpec(BowlingNoFrameskip-v4)\n",
      "EnvSpec(BankHeist-ramNoFrameskip-v0)\n",
      "EnvSpec(KangarooDeterministic-v4)\n",
      "EnvSpec(IceHockey-ramDeterministic-v4)\n",
      "EnvSpec(FrozenLake-v0)\n",
      "EnvSpec(Atlantis-v4)\n",
      "EnvSpec(Assault-v4)\n",
      "EnvSpec(QbertNoFrameskip-v0)\n",
      "EnvSpec(NameThisGameDeterministic-v4)\n",
      "EnvSpec(FishingDerbyDeterministic-v0)\n",
      "EnvSpec(ElevatorActionDeterministic-v0)\n",
      "EnvSpec(AsterixDeterministic-v4)\n",
      "EnvSpec(MsPacman-ram-v4)\n",
      "EnvSpec(Kangaroo-v0)\n",
      "EnvSpec(Assault-ramDeterministic-v4)\n",
      "EnvSpec(MontezumaRevenge-ram-v4)\n",
      "EnvSpec(CarnivalDeterministic-v0)\n",
      "EnvSpec(DemonAttack-ramDeterministic-v4)\n",
      "EnvSpec(OneRoundDeterministicReward-v0)\n",
      "EnvSpec(BowlingDeterministic-v4)\n",
      "EnvSpec(HeroDeterministic-v4)\n",
      "EnvSpec(CarnivalNoFrameskip-v4)\n",
      "EnvSpec(Freeway-v4)\n",
      "EnvSpec(Qbert-ramDeterministic-v4)\n",
      "EnvSpec(FreewayDeterministic-v0)\n",
      "EnvSpec(Berzerk-ramNoFrameskip-v0)\n",
      "EnvSpec(SeaquestDeterministic-v4)\n",
      "EnvSpec(Freeway-v0)\n",
      "EnvSpec(Bowling-ramDeterministic-v0)\n",
      "EnvSpec(Venture-ramNoFrameskip-v4)\n",
      "EnvSpec(OneRoundNondeterministicReward-v0)\n",
      "EnvSpec(FreewayNoFrameskip-v0)\n",
      "EnvSpec(GravitarNoFrameskip-v4)\n",
      "EnvSpec(CrazyClimber-ramNoFrameskip-v0)\n",
      "EnvSpec(Pong-ramDeterministic-v0)\n",
      "EnvSpec(Krull-ram-v0)\n",
      "EnvSpec(PrivateEye-v0)\n",
      "EnvSpec(BoxingDeterministic-v0)\n",
      "EnvSpec(StarGunner-ramDeterministic-v4)\n",
      "EnvSpec(BeamRider-ram-v4)\n",
      "EnvSpec(ChopperCommand-ramNoFrameskip-v4)\n",
      "EnvSpec(Bowling-ramDeterministic-v4)\n",
      "EnvSpec(Pooyan-ramNoFrameskip-v4)\n",
      "EnvSpec(IceHockey-ramNoFrameskip-v0)\n",
      "EnvSpec(PrivateEye-ram-v4)\n",
      "EnvSpec(Zaxxon-ram-v0)\n",
      "EnvSpec(AlienNoFrameskip-v0)\n",
      "EnvSpec(AsterixDeterministic-v0)\n",
      "EnvSpec(Go19x19-v0)\n",
      "EnvSpec(Gopher-v0)\n",
      "EnvSpec(Solaris-ramDeterministic-v4)\n",
      "EnvSpec(YarsRevenge-ram-v0)\n",
      "EnvSpec(PongDeterministic-v0)\n",
      "EnvSpec(GravitarNoFrameskip-v0)\n",
      "EnvSpec(UpNDown-ram-v0)\n",
      "EnvSpec(Gravitar-ramDeterministic-v4)\n",
      "EnvSpec(Jamesbond-ramDeterministic-v4)\n",
      "EnvSpec(TwoRoundNondeterministicReward-v0)\n",
      "EnvSpec(BerzerkDeterministic-v4)\n",
      "EnvSpec(Gravitar-ramNoFrameskip-v4)\n",
      "EnvSpec(Robotank-ramDeterministic-v4)\n",
      "EnvSpec(Centipede-v0)\n",
      "EnvSpec(BattleZone-v4)\n",
      "EnvSpec(Seaquest-ram-v0)\n",
      "EnvSpec(EnduroDeterministic-v0)\n",
      "EnvSpec(VideoPinball-ramNoFrameskip-v0)\n",
      "EnvSpec(KungFuMaster-ramNoFrameskip-v4)\n",
      "EnvSpec(CartPole-v0)\n",
      "EnvSpec(Krull-ramNoFrameskip-v4)\n",
      "EnvSpec(VideoPinball-ram-v4)\n",
      "EnvSpec(Asteroids-ramDeterministic-v4)\n",
      "EnvSpec(Centipede-ramNoFrameskip-v0)\n",
      "EnvSpec(Enduro-ramNoFrameskip-v0)\n",
      "EnvSpec(VentureDeterministic-v0)\n",
      "EnvSpec(ElevatorAction-ramNoFrameskip-v0)\n",
      "EnvSpec(KungFuMasterDeterministic-v0)\n",
      "EnvSpec(ChopperCommandDeterministic-v4)\n",
      "EnvSpec(EnduroNoFrameskip-v4)\n",
      "EnvSpec(Carnival-ramNoFrameskip-v4)\n",
      "EnvSpec(Pooyan-ramDeterministic-v4)\n",
      "EnvSpec(ReversedAddition3-v0)\n",
      "EnvSpec(Qbert-ramDeterministic-v0)\n",
      "EnvSpec(DoubleDunkNoFrameskip-v0)\n",
      "EnvSpec(WizardOfWor-v4)\n",
      "EnvSpec(Walker2d-v1)\n",
      "EnvSpec(Robotank-ramDeterministic-v0)\n",
      "EnvSpec(JourneyEscape-ramNoFrameskip-v0)\n",
      "EnvSpec(WizardOfWor-ramDeterministic-v0)\n",
      "EnvSpec(Bowling-v0)\n",
      "EnvSpec(Berzerk-ramDeterministic-v4)\n",
      "EnvSpec(Tennis-ram-v0)\n",
      "EnvSpec(TimePilot-ramDeterministic-v0)\n",
      "EnvSpec(Go9x9-v0)\n",
      "EnvSpec(Zaxxon-v0)\n",
      "EnvSpec(GuessingGame-v0)\n",
      "EnvSpec(AsteroidsNoFrameskip-v4)\n",
      "EnvSpec(Ant-v1)\n",
      "EnvSpec(AirRaidDeterministic-v4)\n",
      "EnvSpec(Alien-ramNoFrameskip-v4)\n",
      "EnvSpec(TutankhamDeterministic-v0)\n",
      "EnvSpec(FrostbiteDeterministic-v4)\n",
      "EnvSpec(Qbert-ramNoFrameskip-v0)\n",
      "EnvSpec(TimePilot-ram-v4)\n",
      "EnvSpec(PhoenixDeterministic-v4)\n",
      "EnvSpec(Pitfall-ramNoFrameskip-v4)\n",
      "EnvSpec(SeaquestDeterministic-v0)\n",
      "EnvSpec(BeamRiderDeterministic-v0)\n",
      "EnvSpec(Asteroids-ramNoFrameskip-v0)\n",
      "EnvSpec(Phoenix-ram-v4)\n",
      "EnvSpec(RepeatCopy-v0)\n",
      "EnvSpec(MontezumaRevenge-ramNoFrameskip-v4)\n",
      "EnvSpec(StarGunnerNoFrameskip-v4)\n",
      "EnvSpec(ElevatorActionNoFrameskip-v0)\n",
      "EnvSpec(ElevatorAction-ram-v4)\n",
      "EnvSpec(AirRaid-v0)\n",
      "EnvSpec(Pooyan-ramDeterministic-v0)\n",
      "EnvSpec(PrivateEye-v4)\n",
      "EnvSpec(KrullNoFrameskip-v0)\n",
      "EnvSpec(Krull-ram-v4)\n",
      "EnvSpec(CarnivalDeterministic-v4)\n",
      "EnvSpec(OffSwitchCartpole-v0)\n",
      "EnvSpec(BerzerkNoFrameskip-v0)\n",
      "EnvSpec(DoubleDunkDeterministic-v0)\n",
      "EnvSpec(ChopperCommandNoFrameskip-v0)\n",
      "EnvSpec(WizardOfWor-ramNoFrameskip-v4)\n",
      "EnvSpec(Pitfall-ramDeterministic-v0)\n",
      "EnvSpec(Hero-ram-v4)\n",
      "EnvSpec(KungFuMaster-ramDeterministic-v0)\n",
      "EnvSpec(SolarisNoFrameskip-v0)\n",
      "EnvSpec(Reacher-v1)\n",
      "EnvSpec(BankHeist-ramDeterministic-v4)\n",
      "EnvSpec(Atlantis-ram-v0)\n",
      "EnvSpec(Gravitar-ramDeterministic-v0)\n",
      "EnvSpec(IceHockeyDeterministic-v0)\n",
      "EnvSpec(AmidarDeterministic-v0)\n",
      "EnvSpec(DoubleDunk-v4)\n",
      "EnvSpec(Kangaroo-v4)\n",
      "EnvSpec(Alien-v4)\n",
      "EnvSpec(KellyCoinflip-v0)\n",
      "EnvSpec(Bowling-ramNoFrameskip-v0)\n",
      "EnvSpec(BankHeist-ram-v0)\n",
      "EnvSpec(MontezumaRevenge-ramNoFrameskip-v0)\n",
      "EnvSpec(PitfallDeterministic-v4)\n",
      "EnvSpec(Venture-ram-v0)\n",
      "EnvSpec(NameThisGame-v0)\n",
      "EnvSpec(WizardOfWor-ramNoFrameskip-v0)\n",
      "EnvSpec(RoadRunner-ramNoFrameskip-v4)\n",
      "EnvSpec(TutankhamDeterministic-v4)\n",
      "EnvSpec(Carnival-ramDeterministic-v0)\n",
      "EnvSpec(NameThisGameNoFrameskip-v0)\n",
      "EnvSpec(Asterix-ramDeterministic-v0)\n",
      "EnvSpec(YarsRevengeNoFrameskip-v4)\n",
      "EnvSpec(HeroNoFrameskip-v4)\n",
      "EnvSpec(MontezumaRevengeDeterministic-v4)\n",
      "EnvSpec(BattleZone-ramNoFrameskip-v0)\n",
      "EnvSpec(MsPacmanNoFrameskip-v4)\n",
      "EnvSpec(BreakoutNoFrameskip-v0)\n",
      "EnvSpec(Solaris-v0)\n",
      "EnvSpec(ChopperCommand-v4)\n",
      "EnvSpec(StarGunnerNoFrameskip-v0)\n",
      "EnvSpec(Gopher-ramNoFrameskip-v0)\n",
      "EnvSpec(JourneyEscapeNoFrameskip-v4)\n",
      "EnvSpec(SpaceInvadersNoFrameskip-v0)\n",
      "EnvSpec(Krull-ramDeterministic-v4)\n",
      "EnvSpec(Gravitar-v0)\n",
      "EnvSpec(Alien-ram-v4)\n",
      "EnvSpec(Riverraid-ramDeterministic-v0)\n",
      "EnvSpec(EnduroNoFrameskip-v0)\n",
      "EnvSpec(IceHockey-v4)\n",
      "EnvSpec(Seaquest-ramNoFrameskip-v0)\n",
      "EnvSpec(RoadRunner-v0)\n",
      "EnvSpec(SpaceInvadersNoFrameskip-v4)\n",
      "EnvSpec(AirRaid-ramDeterministic-v4)\n",
      "EnvSpec(ElevatorAction-ramDeterministic-v0)\n",
      "EnvSpec(TutankhamNoFrameskip-v0)\n",
      "EnvSpec(AsterixNoFrameskip-v0)\n",
      "EnvSpec(Alien-ram-v0)\n",
      "EnvSpec(Riverraid-ramNoFrameskip-v4)\n",
      "EnvSpec(CrazyClimber-ram-v4)\n",
      "EnvSpec(SkiingDeterministic-v4)\n",
      "EnvSpec(Hero-ram-v0)\n",
      "EnvSpec(RobotankDeterministic-v0)\n",
      "EnvSpec(Breakout-v0)\n",
      "EnvSpec(Venture-ram-v4)\n",
      "EnvSpec(MontezumaRevengeNoFrameskip-v4)\n",
      "EnvSpec(Zaxxon-ramNoFrameskip-v4)\n",
      "EnvSpec(FrostbiteDeterministic-v0)\n",
      "EnvSpec(NameThisGame-ram-v0)\n",
      "EnvSpec(Seaquest-v4)\n",
      "EnvSpec(BerzerkNoFrameskip-v4)\n",
      "EnvSpec(Riverraid-ram-v4)\n",
      "EnvSpec(UpNDown-v0)\n",
      "EnvSpec(Pong-ram-v0)\n",
      "EnvSpec(Pooyan-ram-v0)\n",
      "EnvSpec(Pitfall-v4)\n",
      "EnvSpec(KangarooDeterministic-v0)\n",
      "EnvSpec(RiverraidNoFrameskip-v4)\n",
      "EnvSpec(MontezumaRevengeNoFrameskip-v0)\n",
      "EnvSpec(ChopperCommand-ram-v0)\n",
      "EnvSpec(Reverse-v0)\n",
      "EnvSpec(Frostbite-v0)\n",
      "EnvSpec(Skiing-ram-v0)\n",
      "EnvSpec(CrazyClimberDeterministic-v4)\n",
      "EnvSpec(Centipede-ram-v0)\n",
      "EnvSpec(MsPacman-ramDeterministic-v4)\n",
      "EnvSpec(MountainCarContinuous-v0)\n",
      "EnvSpec(TimePilotDeterministic-v4)\n",
      "EnvSpec(GravitarDeterministic-v4)\n",
      "EnvSpec(Acrobot-v1)\n",
      "EnvSpec(FishingDerbyNoFrameskip-v0)\n",
      "EnvSpec(Breakout-ramDeterministic-v0)\n",
      "EnvSpec(DemonAttack-v0)\n",
      "EnvSpec(DuplicatedInput-v0)\n",
      "EnvSpec(Enduro-ram-v4)\n",
      "EnvSpec(AtlantisDeterministic-v0)\n",
      "EnvSpec(SolarisNoFrameskip-v4)\n",
      "EnvSpec(Pong-ram-v4)\n",
      "EnvSpec(SpaceInvaders-ram-v4)\n",
      "EnvSpec(Hero-ramDeterministic-v4)\n",
      "EnvSpec(Amidar-v4)\n",
      "EnvSpec(YarsRevenge-v0)\n",
      "EnvSpec(Gravitar-ramNoFrameskip-v0)\n",
      "EnvSpec(Jamesbond-v4)\n",
      "EnvSpec(BoxingNoFrameskip-v0)\n",
      "EnvSpec(CartPole-v1)\n",
      "EnvSpec(Assault-ram-v4)\n",
      "EnvSpec(FrostbiteNoFrameskip-v0)\n",
      "EnvSpec(MontezumaRevengeDeterministic-v0)\n",
      "EnvSpec(Berzerk-v4)\n",
      "EnvSpec(BeamRider-ramDeterministic-v0)\n",
      "EnvSpec(Frostbite-v4)\n",
      "EnvSpec(IceHockeyNoFrameskip-v4)\n",
      "EnvSpec(Bowling-v4)\n",
      "EnvSpec(KungFuMaster-v4)\n",
      "EnvSpec(Skiing-ramDeterministic-v4)\n",
      "EnvSpec(PooyanNoFrameskip-v0)\n",
      "EnvSpec(AirRaid-v4)\n",
      "EnvSpec(Tutankham-ram-v0)\n",
      "EnvSpec(Asteroids-ramDeterministic-v0)\n",
      "EnvSpec(Tutankham-v4)\n",
      "EnvSpec(PrivateEyeNoFrameskip-v4)\n",
      "EnvSpec(Amidar-ramDeterministic-v0)\n",
      "EnvSpec(Berzerk-ram-v0)\n",
      "EnvSpec(DoubleDunk-ram-v0)\n",
      "EnvSpec(FreewayNoFrameskip-v4)\n",
      "EnvSpec(TennisDeterministic-v4)\n",
      "EnvSpec(DoubleDunk-v0)\n",
      "EnvSpec(Asterix-ramNoFrameskip-v4)\n",
      "EnvSpec(Asterix-ramDeterministic-v4)\n",
      "EnvSpec(ChopperCommandNoFrameskip-v4)\n",
      "EnvSpec(PrivateEyeDeterministic-v4)\n",
      "EnvSpec(StarGunner-v4)\n",
      "EnvSpec(PrivateEye-ramDeterministic-v4)\n",
      "EnvSpec(SeaquestNoFrameskip-v4)\n",
      "EnvSpec(GopherNoFrameskip-v0)\n",
      "EnvSpec(SolarisDeterministic-v4)\n",
      "EnvSpec(BeamRider-ram-v0)\n",
      "EnvSpec(RobotankDeterministic-v4)\n",
      "EnvSpec(Solaris-v4)\n",
      "EnvSpec(SolarisDeterministic-v0)\n",
      "EnvSpec(ElevatorAction-v0)\n",
      "EnvSpec(DoubleDunk-ram-v4)\n",
      "EnvSpec(BankHeistDeterministic-v0)\n",
      "EnvSpec(AsteroidsDeterministic-v0)\n",
      "EnvSpec(Atlantis-ram-v4)\n",
      "EnvSpec(Blackjack-v0)\n",
      "EnvSpec(Alien-v0)\n",
      "EnvSpec(Atlantis-ramNoFrameskip-v4)\n",
      "EnvSpec(FishingDerbyNoFrameskip-v4)\n",
      "EnvSpec(Kangaroo-ramDeterministic-v0)\n",
      "EnvSpec(TimePilotDeterministic-v0)\n",
      "EnvSpec(YarsRevengeDeterministic-v0)\n",
      "EnvSpec(StarGunner-ramNoFrameskip-v4)\n",
      "EnvSpec(SeaquestNoFrameskip-v0)\n",
      "EnvSpec(KungFuMasterNoFrameskip-v4)\n",
      "EnvSpec(FishingDerby-ram-v4)\n",
      "EnvSpec(Qbert-v4)\n",
      "EnvSpec(MountainCar-v0)\n",
      "EnvSpec(FishingDerby-ramDeterministic-v4)\n",
      "EnvSpec(Enduro-v4)\n",
      "EnvSpec(Centipede-ram-v4)\n",
      "EnvSpec(RiverraidNoFrameskip-v0)\n",
      "EnvSpec(Qbert-v0)\n",
      "EnvSpec(JourneyEscape-v4)\n",
      "EnvSpec(AtlantisNoFrameskip-v4)\n",
      "EnvSpec(Riverraid-v0)\n",
      "EnvSpec(Pong-ramNoFrameskip-v4)\n",
      "EnvSpec(FishingDerby-ramNoFrameskip-v0)\n",
      "EnvSpec(Gopher-ram-v0)\n",
      "EnvSpec(Seaquest-ramDeterministic-v0)\n",
      "EnvSpec(TimePilot-v4)\n",
      "EnvSpec(Solaris-ramDeterministic-v0)\n",
      "EnvSpec(DemonAttackDeterministic-v4)\n",
      "EnvSpec(VideoPinballDeterministic-v0)\n",
      "EnvSpec(StarGunner-ramNoFrameskip-v0)\n",
      "EnvSpec(Asteroids-ramNoFrameskip-v4)\n",
      "EnvSpec(LunarLanderContinuous-v2)\n",
      "EnvSpec(Zaxxon-ramDeterministic-v4)\n",
      "EnvSpec(YarsRevenge-v4)\n",
      "EnvSpec(Swimmer-v1)\n",
      "EnvSpec(AirRaid-ramDeterministic-v0)\n",
      "EnvSpec(UpNDown-ramDeterministic-v0)\n",
      "EnvSpec(AsterixNoFrameskip-v4)\n",
      "EnvSpec(ChopperCommandDeterministic-v0)\n",
      "EnvSpec(Seaquest-ram-v4)\n",
      "EnvSpec(PooyanNoFrameskip-v4)\n",
      "EnvSpec(Kangaroo-ramDeterministic-v4)\n",
      "EnvSpec(SpaceInvaders-v4)\n",
      "EnvSpec(Seaquest-ramNoFrameskip-v4)\n",
      "EnvSpec(Jamesbond-v0)\n",
      "EnvSpec(TimePilot-ramDeterministic-v4)\n",
      "EnvSpec(Tutankham-ramNoFrameskip-v0)\n",
      "EnvSpec(AssaultNoFrameskip-v4)\n",
      "EnvSpec(RobotankNoFrameskip-v4)\n",
      "EnvSpec(MontezumaRevenge-ramDeterministic-v4)\n",
      "EnvSpec(Solaris-ram-v0)\n",
      "EnvSpec(Amidar-ramDeterministic-v4)\n",
      "EnvSpec(Robotank-v4)\n",
      "EnvSpec(NameThisGame-ramNoFrameskip-v4)\n",
      "EnvSpec(CNNClassifierTraining-v0)\n",
      "EnvSpec(Pooyan-ramNoFrameskip-v0)\n",
      "EnvSpec(SpaceInvadersDeterministic-v0)\n",
      "EnvSpec(DemonAttackNoFrameskip-v0)\n",
      "EnvSpec(Gravitar-ram-v4)\n",
      "EnvSpec(Atlantis-v0)\n",
      "EnvSpec(Qbert-ramNoFrameskip-v4)\n",
      "EnvSpec(KungFuMaster-ramDeterministic-v4)\n",
      "EnvSpec(VideoPinball-ram-v0)\n",
      "EnvSpec(Tutankham-ramNoFrameskip-v4)\n",
      "EnvSpec(Breakout-ram-v0)\n",
      "EnvSpec(PrivateEyeNoFrameskip-v0)\n",
      "EnvSpec(NameThisGame-ramDeterministic-v4)\n",
      "EnvSpec(WizardOfWor-ram-v0)\n",
      "EnvSpec(BattleZoneDeterministic-v4)\n",
      "EnvSpec(Tutankham-ramDeterministic-v0)\n",
      "EnvSpec(BowlingNoFrameskip-v0)\n",
      "EnvSpec(UpNDownNoFrameskip-v4)\n",
      "EnvSpec(Berzerk-v0)\n",
      "EnvSpec(Hex9x9-v0)\n",
      "EnvSpec(Asterix-ram-v4)\n",
      "EnvSpec(DemonAttack-ramNoFrameskip-v0)\n",
      "EnvSpec(HeroNoFrameskip-v0)\n",
      "EnvSpec(CrazyClimber-ramDeterministic-v0)\n",
      "EnvSpec(MontezumaRevenge-v0)\n",
      "EnvSpec(StarGunnerDeterministic-v4)\n",
      "EnvSpec(IceHockey-v0)\n",
      "EnvSpec(Venture-v4)\n",
      "EnvSpec(Atlantis-ramDeterministic-v4)\n",
      "EnvSpec(Phoenix-ram-v0)\n",
      "EnvSpec(BipedalWalker-v2)\n",
      "EnvSpec(Venture-ramDeterministic-v4)\n",
      "EnvSpec(AirRaid-ramNoFrameskip-v4)\n",
      "EnvSpec(CarnivalNoFrameskip-v0)\n",
      "EnvSpec(TimePilot-v0)\n",
      "EnvSpec(KungFuMaster-ram-v4)\n",
      "EnvSpec(ElevatorAction-ramDeterministic-v4)\n",
      "EnvSpec(Alien-ramDeterministic-v4)\n",
      "EnvSpec(Carnival-v4)\n",
      "EnvSpec(BattleZoneDeterministic-v0)\n",
      "EnvSpec(Carnival-ram-v4)\n",
      "EnvSpec(Robotank-ram-v4)\n",
      "EnvSpec(DoubleDunkNoFrameskip-v4)\n",
      "EnvSpec(Venture-v0)\n",
      "EnvSpec(BipedalWalkerHardcore-v2)\n",
      "EnvSpec(Zaxxon-ram-v4)\n",
      "EnvSpec(Kangaroo-ramNoFrameskip-v4)\n",
      "EnvSpec(Enduro-ramDeterministic-v0)\n",
      "EnvSpec(Skiing-ram-v4)\n",
      "EnvSpec(KungFuMaster-ram-v0)\n",
      "EnvSpec(FreewayDeterministic-v4)\n",
      "EnvSpec(Boxing-ram-v4)\n",
      "EnvSpec(SemisuperPendulumRandom-v0)\n",
      "EnvSpec(Robotank-v0)\n",
      "EnvSpec(Boxing-ramNoFrameskip-v0)\n",
      "EnvSpec(MontezumaRevenge-ram-v0)\n",
      "EnvSpec(AtlantisDeterministic-v4)\n",
      "EnvSpec(MsPacman-ram-v0)\n",
      "EnvSpec(VideoPinball-ramDeterministic-v4)\n",
      "EnvSpec(Skiing-v0)\n",
      "EnvSpec(Asterix-ramNoFrameskip-v0)\n",
      "EnvSpec(TennisNoFrameskip-v0)\n",
      "EnvSpec(Jamesbond-ramDeterministic-v0)\n",
      "EnvSpec(KangarooNoFrameskip-v0)\n",
      "EnvSpec(JourneyEscape-ram-v0)\n",
      "EnvSpec(KrullDeterministic-v4)\n",
      "EnvSpec(Pitfall-ramDeterministic-v4)\n",
      "EnvSpec(Tutankham-ramDeterministic-v4)\n",
      "EnvSpec(RoadRunner-ram-v4)\n",
      "EnvSpec(Zaxxon-ramNoFrameskip-v0)\n",
      "EnvSpec(BankHeist-v4)\n",
      "EnvSpec(QbertDeterministic-v0)\n",
      "EnvSpec(BankHeist-v0)\n",
      "EnvSpec(CliffWalking-v0)\n",
      "EnvSpec(Hopper-v1)\n",
      "EnvSpec(Jamesbond-ram-v4)\n",
      "EnvSpec(StarGunnerDeterministic-v0)\n",
      "EnvSpec(VideoPinballNoFrameskip-v0)\n",
      "EnvSpec(Skiing-ramDeterministic-v0)\n",
      "EnvSpec(MsPacman-v4)\n",
      "EnvSpec(GopherDeterministic-v4)\n",
      "EnvSpec(UpNDown-ramDeterministic-v4)\n",
      "EnvSpec(Phoenix-ramNoFrameskip-v4)\n",
      "EnvSpec(Zaxxon-v4)\n",
      "EnvSpec(Berzerk-ramDeterministic-v0)\n",
      "EnvSpec(Asteroids-v0)\n",
      "EnvSpec(Enduro-ramNoFrameskip-v4)\n",
      "EnvSpec(RoadRunnerDeterministic-v4)\n",
      "EnvSpec(BankHeist-ramDeterministic-v0)\n",
      "EnvSpec(DoubleDunkDeterministic-v4)\n",
      "EnvSpec(CentipedeDeterministic-v4)\n",
      "EnvSpec(BattleZone-v0)\n",
      "EnvSpec(ChopperCommand-ram-v4)\n",
      "EnvSpec(Frostbite-ramDeterministic-v4)\n",
      "EnvSpec(MontezumaRevenge-v4)\n",
      "EnvSpec(PitfallNoFrameskip-v4)\n",
      "EnvSpec(Amidar-ramNoFrameskip-v4)\n",
      "EnvSpec(PhoenixNoFrameskip-v4)\n",
      "EnvSpec(MsPacman-ramDeterministic-v0)\n",
      "EnvSpec(Frostbite-ramNoFrameskip-v4)\n",
      "EnvSpec(Amidar-ram-v0)\n",
      "EnvSpec(JourneyEscapeNoFrameskip-v0)\n",
      "EnvSpec(GravitarDeterministic-v0)\n",
      "EnvSpec(YarsRevenge-ramDeterministic-v0)\n",
      "EnvSpec(SemisuperPendulumDecay-v0)\n",
      "EnvSpec(CrazyClimberNoFrameskip-v4)\n",
      "EnvSpec(ElevatorActionDeterministic-v4)\n",
      "EnvSpec(MsPacmanNoFrameskip-v0)\n",
      "EnvSpec(ChopperCommand-v0)\n",
      "EnvSpec(SpaceInvaders-ramDeterministic-v4)\n",
      "EnvSpec(JourneyEscape-ram-v4)\n",
      "EnvSpec(Tennis-ram-v4)\n",
      "EnvSpec(VideoPinballDeterministic-v4)\n",
      "EnvSpec(Freeway-ram-v4)\n",
      "EnvSpec(AirRaid-ramNoFrameskip-v0)\n",
      "EnvSpec(BankHeistNoFrameskip-v4)\n",
      "EnvSpec(KrullDeterministic-v0)\n",
      "EnvSpec(Freeway-ramNoFrameskip-v0)\n",
      "EnvSpec(UpNDownDeterministic-v4)\n",
      "EnvSpec(Asterix-v0)\n",
      "EnvSpec(BankHeistNoFrameskip-v0)\n",
      "EnvSpec(SpaceInvaders-ramNoFrameskip-v4)\n",
      "EnvSpec(WizardOfWor-ram-v4)\n",
      "EnvSpec(Venture-ramDeterministic-v0)\n",
      "EnvSpec(CrazyClimber-v4)\n",
      "EnvSpec(PooyanDeterministic-v0)\n",
      "EnvSpec(Alien-ramDeterministic-v0)\n",
      "EnvSpec(BeamRider-ramNoFrameskip-v0)\n",
      "EnvSpec(JourneyEscape-ramDeterministic-v0)\n",
      "EnvSpec(Pooyan-ram-v4)\n",
      "EnvSpec(AirRaidDeterministic-v0)\n",
      "EnvSpec(Robotank-ram-v0)\n",
      "EnvSpec(Pitfall-ramNoFrameskip-v0)\n",
      "EnvSpec(Gravitar-ram-v0)\n",
      "EnvSpec(SpaceInvaders-ram-v0)\n",
      "EnvSpec(DoubleDunk-ramDeterministic-v4)\n",
      "EnvSpec(InvertedPendulum-v1)\n",
      "EnvSpec(StarGunner-v0)\n",
      "EnvSpec(ReversedAddition-v0)\n",
      "EnvSpec(TimePilot-ram-v0)\n",
      "EnvSpec(BeamRiderNoFrameskip-v0)\n",
      "EnvSpec(Gopher-v4)\n",
      "EnvSpec(KellyCoinflipGeneralized-v0)\n",
      "EnvSpec(Jamesbond-ramNoFrameskip-v4)\n",
      "EnvSpec(Phoenix-ramDeterministic-v4)\n",
      "EnvSpec(Hero-v0)\n",
      "EnvSpec(Skiing-v4)\n",
      "EnvSpec(Carnival-ramNoFrameskip-v0)\n",
      "EnvSpec(FishingDerbyDeterministic-v4)\n",
      "EnvSpec(Asterix-ram-v0)\n",
      "EnvSpec(ZaxxonDeterministic-v0)\n",
      "EnvSpec(Solaris-ramNoFrameskip-v4)\n",
      "EnvSpec(Hero-ramNoFrameskip-v4)\n",
      "EnvSpec(BreakoutDeterministic-v0)\n",
      "EnvSpec(RoadRunnerDeterministic-v0)\n",
      "EnvSpec(Pong-v4)\n",
      "EnvSpec(PongDeterministic-v4)\n",
      "EnvSpec(HotterColder-v0)\n",
      "EnvSpec(Gopher-ram-v4)\n",
      "EnvSpec(ZaxxonDeterministic-v4)\n",
      "EnvSpec(StarGunner-ram-v4)\n",
      "EnvSpec(IceHockeyNoFrameskip-v0)\n",
      "EnvSpec(WizardOfWor-v0)\n",
      "EnvSpec(Freeway-ramDeterministic-v0)\n",
      "EnvSpec(NameThisGame-ram-v4)\n",
      "EnvSpec(BattleZone-ram-v4)\n",
      "EnvSpec(Pitfall-ram-v0)\n",
      "EnvSpec(NChain-v0)\n",
      "EnvSpec(Riverraid-ramDeterministic-v4)\n",
      "EnvSpec(Copy-v0)\n",
      "EnvSpec(KungFuMasterNoFrameskip-v0)\n",
      "EnvSpec(Qbert-ram-v0)\n",
      "EnvSpec(CentipedeNoFrameskip-v0)\n",
      "EnvSpec(Gopher-ramDeterministic-v0)\n",
      "EnvSpec(AsteroidsDeterministic-v4)\n",
      "EnvSpec(Boxing-v0)\n",
      "EnvSpec(VentureDeterministic-v4)\n",
      "EnvSpec(UpNDownDeterministic-v0)\n",
      "EnvSpec(Asteroids-ram-v4)\n",
      "EnvSpec(PredictActionsCartpole-v0)\n",
      "EnvSpec(Pitfall-ram-v4)\n",
      "EnvSpec(SpaceInvadersDeterministic-v4)\n",
      "EnvSpec(FrostbiteNoFrameskip-v4)\n",
      "EnvSpec(DoubleDunk-ramNoFrameskip-v0)\n",
      "EnvSpec(TutankhamNoFrameskip-v4)\n",
      "EnvSpec(Tennis-v4)\n",
      "EnvSpec(TennisDeterministic-v0)\n",
      "EnvSpec(Kangaroo-ram-v4)\n",
      "EnvSpec(Enduro-ramDeterministic-v4)\n",
      "EnvSpec(KangarooNoFrameskip-v4)\n",
      "EnvSpec(AssaultDeterministic-v4)\n",
      "EnvSpec(WizardOfWorDeterministic-v4)\n",
      "EnvSpec(AlienDeterministic-v4)\n",
      "EnvSpec(Breakout-ramNoFrameskip-v4)\n",
      "EnvSpec(YarsRevenge-ramNoFrameskip-v0)\n",
      "EnvSpec(Alien-ramNoFrameskip-v0)\n",
      "EnvSpec(YarsRevenge-ram-v4)\n",
      "EnvSpec(BankHeistDeterministic-v4)\n",
      "EnvSpec(StarGunner-ramDeterministic-v0)\n",
      "EnvSpec(FishingDerby-v0)\n",
      "EnvSpec(UpNDown-ramNoFrameskip-v4)\n",
      "EnvSpec(VideoPinball-v4)\n",
      "EnvSpec(Frostbite-ram-v0)\n",
      "EnvSpec(UpNDown-ramNoFrameskip-v0)\n",
      "EnvSpec(Amidar-ramNoFrameskip-v0)\n",
      "EnvSpec(JamesbondNoFrameskip-v0)\n",
      "EnvSpec(AlienDeterministic-v0)\n",
      "EnvSpec(YarsRevenge-ramDeterministic-v4)\n",
      "EnvSpec(Phoenix-ramDeterministic-v0)\n",
      "EnvSpec(MsPacman-ramNoFrameskip-v0)\n",
      "EnvSpec(Krull-ramNoFrameskip-v0)\n",
      "EnvSpec(WizardOfWorDeterministic-v0)\n",
      "EnvSpec(Seaquest-ramDeterministic-v4)\n",
      "EnvSpec(Centipede-ramNoFrameskip-v4)\n",
      "EnvSpec(PrivateEye-ramNoFrameskip-v0)\n",
      "EnvSpec(Seaquest-v0)\n",
      "EnvSpec(Skiing-ramNoFrameskip-v4)\n",
      "EnvSpec(RoadRunner-ram-v0)\n",
      "EnvSpec(Tennis-ramDeterministic-v0)\n",
      "EnvSpec(QbertNoFrameskip-v4)\n",
      "EnvSpec(BankHeist-ram-v4)\n",
      "EnvSpec(PrivateEye-ram-v0)\n",
      "EnvSpec(VentureNoFrameskip-v0)\n",
      "EnvSpec(MontezumaRevenge-ramDeterministic-v0)\n",
      "EnvSpec(Bowling-ram-v0)\n",
      "EnvSpec(ElevatorAction-ramNoFrameskip-v4)\n",
      "EnvSpec(Boxing-v4)\n",
      "EnvSpec(UpNDownNoFrameskip-v0)\n",
      "EnvSpec(Assault-ramNoFrameskip-v4)\n",
      "EnvSpec(GopherNoFrameskip-v4)\n",
      "EnvSpec(Qbert-ram-v4)\n",
      "EnvSpec(Krull-v4)\n",
      "EnvSpec(Centipede-v4)\n",
      "EnvSpec(Pendulum-v0)\n",
      "EnvSpec(BattleZoneNoFrameskip-v4)\n",
      "EnvSpec(RobotankNoFrameskip-v0)\n",
      "EnvSpec(HeroDeterministic-v0)\n",
      "EnvSpec(Tennis-ramNoFrameskip-v4)\n",
      "EnvSpec(DemonAttack-ramNoFrameskip-v4)\n",
      "EnvSpec(FishingDerby-ramNoFrameskip-v4)\n",
      "EnvSpec(Boxing-ramNoFrameskip-v4)\n",
      "EnvSpec(MsPacmanDeterministic-v0)\n",
      "EnvSpec(TimePilot-ramNoFrameskip-v4)\n",
      "EnvSpec(YarsRevengeDeterministic-v4)\n",
      "EnvSpec(WizardOfWor-ramDeterministic-v4)\n",
      "EnvSpec(MsPacmanDeterministic-v4)\n",
      "EnvSpec(JamesbondDeterministic-v4)\n",
      "EnvSpec(Pusher-v0)\n",
      "EnvSpec(RoadRunnerNoFrameskip-v4)\n",
      "EnvSpec(SpaceInvaders-ramNoFrameskip-v0)\n",
      "EnvSpec(Krull-ramDeterministic-v0)\n",
      "EnvSpec(Enduro-v0)\n",
      "EnvSpec(AmidarNoFrameskip-v4)\n",
      "EnvSpec(KungFuMasterDeterministic-v4)\n",
      "EnvSpec(RoadRunner-ramNoFrameskip-v0)\n",
      "EnvSpec(DoubleDunk-ramDeterministic-v0)\n",
      "EnvSpec(Skiing-ramNoFrameskip-v0)\n",
      "EnvSpec(FishingDerby-v4)\n",
      "EnvSpec(VideoPinball-v0)\n",
      "EnvSpec(BeamRiderNoFrameskip-v4)\n",
      "EnvSpec(YarsRevenge-ramNoFrameskip-v4)\n",
      "EnvSpec(PongNoFrameskip-v0)\n",
      "EnvSpec(AirRaidNoFrameskip-v0)\n",
      "EnvSpec(PrivateEye-ramDeterministic-v0)\n",
      "EnvSpec(BattleZoneNoFrameskip-v0)\n",
      "EnvSpec(Boxing-ramDeterministic-v0)\n",
      "EnvSpec(JamesbondNoFrameskip-v4)\n",
      "EnvSpec(IceHockey-ramDeterministic-v0)\n",
      "EnvSpec(PooyanDeterministic-v4)\n",
      "EnvSpec(Frostbite-ramNoFrameskip-v0)\n",
      "EnvSpec(Riverraid-ram-v0)\n",
      "EnvSpec(BreakoutNoFrameskip-v4)\n",
      "EnvSpec(Tutankham-v0)\n",
      "EnvSpec(BankHeist-ramNoFrameskip-v4)\n",
      "EnvSpec(Atlantis-ramDeterministic-v0)\n",
      "EnvSpec(LunarLander-v2)\n",
      "EnvSpec(CrazyClimberNoFrameskip-v0)\n",
      "EnvSpec(Pitfall-v0)\n",
      "EnvSpec(Frostbite-ramDeterministic-v0)\n",
      "EnvSpec(Asterix-v4)\n",
      "EnvSpec(PhoenixDeterministic-v0)\n",
      "EnvSpec(Asteroids-ram-v0)\n",
      "EnvSpec(CrazyClimber-v0)\n",
      "EnvSpec(PrivateEyeDeterministic-v0)\n",
      "EnvSpec(Atlantis-ramNoFrameskip-v0)\n",
      "EnvSpec(DemonAttackNoFrameskip-v4)\n",
      "EnvSpec(BattleZone-ramDeterministic-v4)\n",
      "EnvSpec(Centipede-ramDeterministic-v4)\n",
      "EnvSpec(AssaultNoFrameskip-v0)\n",
      "EnvSpec(CrazyClimber-ramDeterministic-v4)\n",
      "EnvSpec(ChopperCommand-ramDeterministic-v4)\n",
      "EnvSpec(KungFuMaster-ramNoFrameskip-v0)\n"
     ]
    }
   ],
   "source": [
    "for e in list(all_env):\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Play the cartploe game with stochastic control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as anm\n",
    "from matplotlib import rc\n",
    "#rc('animation', html='html5')\n",
    "rc('animation', ffmpeg_path='/usr/bin/ffmpeg')\n",
    "%matplotlib inline\n",
    "\n",
    "from matplotlib import animation\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "\n",
    "def env_render(env_vis):\n",
    "    plt.figure()\n",
    "    plot = plt.imshow(env_vis[0])\n",
    "    plt.axis('off')\n",
    "    def animate(i):\n",
    "        plot.set_data(env_vis[i])\n",
    "\n",
    "    anim = anm.FuncAnimation(plt.gcf(),\n",
    "                             animate,\n",
    "                             frames=len(env_vis),\n",
    "                             interval=20,\n",
    "                             repeat=True,\n",
    "                             repeat_delay=20)\n",
    "    display(display_animation(anim, default_mode='loop'))\n",
    "    \n",
    "env = gym.make('CartPole-v0')\n",
    "n_episodes = 1\n",
    "env_vis = []\n",
    "for i_episode in range(n_episodes):\n",
    "    observation = env.reset()\n",
    "    for t in range(100):\n",
    "        env_vis.append(env.render(mode = 'rgb_array'))\n",
    "        print(observation)\n",
    "        action = env.action_space.sample()\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        if done:\n",
    "            print(\"Episode finished at t{}\".format(t+1))\n",
    "            break\n",
    "env.render(close=True)\n",
    "env_render(env_vis)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Policies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy:policy_random, Min reward:9.0, Max reward:81.0, Average reward:22.52\n",
      "Policy:policy_logic, Min reward:25.0, Max reward:58.0, Average reward:42.37\n"
     ]
    }
   ],
   "source": [
    "def policy_logic(env,obs):\n",
    "    return 1 if obs[2] > 0 else 0\n",
    "def policy_random(env,obs):\n",
    "    return env.action_space.sample()\n",
    "\n",
    "def experiment(policy, n_episodes, rewards_max):\n",
    "    rewards=np.empty(shape=(n_episodes))\n",
    "    env = gym.make('CartPole-v0')\n",
    "    \n",
    "    for i in range(n_episodes):\n",
    "        obs = env.reset()\n",
    "        done = False\n",
    "        episode_reward = 0\n",
    "        while not done:\n",
    "            action = policy(env,obs)\n",
    "            obs, reward, done, info = env.step(action)\n",
    "            episode_reward += reward\n",
    "            if episode_reward > rewards_max:\n",
    "                break\n",
    "        rewards[i]=episode_reward\n",
    "        #print(\"Episode finished at t{}\".format(reward))\n",
    "    print('Policy:{}, Min reward:{}, Max reward:{}, Average reward:{}'\n",
    "          .format(policy.__name__,\n",
    "                  np.min(rewards),\n",
    "                  np.max(rewards),\n",
    "                  np.mean(rewards)))\n",
    "\n",
    "n_episodes = 100\n",
    "rewards_max = 10000\n",
    "experiment(policy_random, n_episodes, rewards_max)\n",
    "experiment(policy_logic, n_episodes, rewards_max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Policies with Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy:policy_random, Min reward:8.0, Max reward:200.0, Average reward:36.52\n",
      "Policy:policy_logic, Min reward:25.0, Max reward:66.0, Average reward:44.18\n"
     ]
    }
   ],
   "source": [
    "def policy_logic(theta,obs):\n",
    "    # just ignore theta\n",
    "    return 1 if obs[2] > 0 else 0\n",
    "\n",
    "def policy_random(theta,obs):\n",
    "    return 0 if np.matmul(theta,obs) < 0 else 1\n",
    "\n",
    "def episode(env, policy, rewards_max):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    episode_reward = 0\n",
    "    if policy.__name__ in ['policy_random']:\n",
    "        theta = np.random.rand(4) * 2 - 1\n",
    "    else:\n",
    "        theta = None\n",
    "    while not done:\n",
    "        action = policy(theta,obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if episode_reward > rewards_max:\n",
    "            break\n",
    "    return episode_reward\n",
    "    \n",
    "def experiment(policy, n_episodes, rewards_max):\n",
    "    rewards=np.empty(shape=(n_episodes))\n",
    "    env = gym.make('CartPole-v0')\n",
    "    \n",
    "    for i in range(n_episodes):\n",
    "        rewards[i]=episode(env,policy,rewards_max)\n",
    "        #print(\"Episode finished at t{}\".format(reward))\n",
    "    print('Policy:{}, Min reward:{}, Max reward:{}, Average reward:{}'\n",
    "          .format(policy.__name__,\n",
    "                  np.min(rewards),\n",
    "                  np.max(rewards),\n",
    "                  np.mean(rewards)))\n",
    "\n",
    "n_episodes = 100\n",
    "rewards_max = 10000\n",
    "experiment(policy_random, n_episodes, rewards_max)\n",
    "experiment(policy_logic, n_episodes, rewards_max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Policies with Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trained theta: [-0.1526904   0.29178823 -0.12482558  0.783546  ], rewards: 200.0\n",
      "Policy:policy_random, Min reward:200.0, Max reward:200.0, Average reward:200.0\n",
      "Policy:policy_logic, Min reward:24.0, Max reward:68.0, Average reward:41.66\n"
     ]
    }
   ],
   "source": [
    "# train with random search\n",
    "\n",
    "def policy_logic(theta,obs):\n",
    "    # just ignore theta\n",
    "    return 1 if obs[2] > 0 else 0\n",
    "\n",
    "def policy_random(theta,obs):\n",
    "    return 0 if np.matmul(theta,obs) < 0 else 1\n",
    "\n",
    "def episode(env,policy, rewards_max,theta):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    episode_reward = 0\n",
    "\n",
    "    while not done:\n",
    "        action = policy(theta,obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if episode_reward > rewards_max:\n",
    "            break\n",
    "    return episode_reward\n",
    "\n",
    "def train(policy, n_episodes, rewards_max):\n",
    "\n",
    "    env = gym.make('CartPole-v0')\n",
    "    np.random.seed(0)\n",
    "    \n",
    "    theta_best = np.empty(shape=[4])\n",
    "    reward_best = 0\n",
    "\n",
    "    for i in range(n_episodes):\n",
    "        if policy.__name__ in ['policy_random']:\n",
    "            theta = np.random.rand(4) * 2 - 1\n",
    "        else:\n",
    "            theta = None\n",
    "        \n",
    "        reward_episode=episode(env,policy,rewards_max, theta)\n",
    "        if reward_episode > reward_best:\n",
    "            reward_best = reward_episode\n",
    "            theta_best = theta.copy()\n",
    "    return reward_best,theta_best\n",
    "    \n",
    "def experiment(policy, n_episodes, rewards_max, theta=None):\n",
    "    rewards=np.empty(shape=[n_episodes])\n",
    "    env = gym.make('CartPole-v0')\n",
    "    \n",
    "    for i in range(n_episodes):\n",
    "        rewards[i]=episode(env,policy,rewards_max,theta)\n",
    "        #print(\"Episode finished at t{}\".format(reward))\n",
    "    print('Policy:{}, Min reward:{}, Max reward:{}, Average reward:{}'\n",
    "          .format(policy.__name__,\n",
    "                  np.min(rewards),\n",
    "                  np.max(rewards),\n",
    "                  np.mean(rewards)))\n",
    "\n",
    "n_episodes = 100\n",
    "rewards_max = 10000\n",
    "\n",
    "reward,theta = train(policy_random, n_episodes, rewards_max)\n",
    "print('trained theta: {}, rewards: {}'.format(theta,reward))\n",
    "experiment(policy_random, n_episodes, rewards_max, theta)\n",
    "\n",
    "experiment(policy_logic, n_episodes, rewards_max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Policies with Training until Certain Rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trained theta: [-0.1526904   0.29178823 -0.12482558  0.783546  ], rewards: 200.0\n",
      "Policy:policy_random, Min reward:200.0, Max reward:200.0, Average reward:200.0\n",
      "Policy:policy_logic, Min reward:24.0, Max reward:64.0, Average reward:41.96\n"
     ]
    }
   ],
   "source": [
    "# train with random search until we reach rewards > 200\n",
    "def policy_logic(theta,obs):\n",
    "    # just ignore theta\n",
    "    return 1 if obs[2] > 0 else 0\n",
    "\n",
    "def policy_random(theta,obs):\n",
    "    return 0 if np.matmul(theta,obs) < 0 else 1\n",
    "\n",
    "def episode(env,policy, rewards_max,theta):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    episode_reward = 0\n",
    "\n",
    "    while not done:\n",
    "        action = policy(theta,obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if episode_reward > rewards_max:\n",
    "            break\n",
    "    return episode_reward\n",
    "\n",
    "def train(policy, n_episodes, rewards_max):\n",
    "\n",
    "    env = gym.make('CartPole-v0')\n",
    "    np.random.seed(0)\n",
    "    \n",
    "    theta_best = np.empty(shape=[4])\n",
    "    reward_best = 0\n",
    "\n",
    "    # n_episodes acts as a max in this case\n",
    "    for i in range(n_episodes):\n",
    "        if policy.__name__ in ['policy_random']:\n",
    "            theta = np.random.rand(4) * 2 - 1\n",
    "        else:\n",
    "            theta = None\n",
    "        reward_episode=episode(env,policy,rewards_max, theta)\n",
    "        if reward_episode > reward_best:\n",
    "            reward_best = reward_episode\n",
    "            theta_best = theta.copy()\n",
    "        if reward_best >= rewards_max:\n",
    "            break\n",
    "    return reward_best,theta_best\n",
    "    \n",
    "def experiment(policy, n_episodes, rewards_max, theta=None):\n",
    "    rewards=np.empty(shape=[n_episodes])\n",
    "    env = gym.make('CartPole-v0')\n",
    "    \n",
    "    for i in range(n_episodes):\n",
    "        rewards[i]=episode(env,policy,rewards_max,theta)\n",
    "        #print(\"Episode finished at t{}\".format(reward))\n",
    "    print('Policy:{}, Min reward:{}, Max reward:{}, Average reward:{}'\n",
    "          .format(policy.__name__,\n",
    "                  np.min(rewards),\n",
    "                  np.max(rewards),\n",
    "                  np.mean(rewards)))\n",
    "\n",
    "n_episodes = 10000 # acts as max episodes to use in training\n",
    "rewards_max = 200\n",
    "reward,theta = train(policy_random, n_episodes, rewards_max)\n",
    "print('trained theta: {}, rewards: {}'.format(theta,reward))\n",
    "\n",
    "n_episodes = 100\n",
    "rewards_max = 10000\n",
    "experiment(policy_random, n_episodes, rewards_max, theta)\n",
    "experiment(policy_logic, n_episodes, rewards_max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train with the neural network\n",
    "\n",
    "def policy_random(theta,obs):\n",
    "    return 0 if np.matmul(theta,obs) < 0 else 1\n",
    "\n",
    "def policy_naive_nn(nn,obs):\n",
    "    return np.argmax(nn.predict(np.array([obs])))\n",
    "\n",
    "\n",
    "# returns obs -> actions -> rewards arrays\n",
    "# specify t_max to run for t_max steps\n",
    "# specify r_max to run until r_max is reached\n",
    "# specify both t_max ad r_max to run for t_max but break if r_max is reached\n",
    "def episode(env, policy, theta, r_max=0, t_max=0, return_hist_reward=0):\n",
    "    if return_hist_reward > 0:\n",
    "        o_list=[]\n",
    "        a_list=[]\n",
    "        r_list=[]\n",
    "    \n",
    "    episode_reward = 0\n",
    "\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    t = 0\n",
    "    while not done:\n",
    "        action = policy(theta,obs)\n",
    "        if return_hist_reward>0:\n",
    "            o_list.append(obs)\n",
    "            a_list.append(action)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        if return_hist_reward>0:\n",
    "            r_list.append(reward)\n",
    "        episode_reward += reward\n",
    "        if r_max > 0 and episode_reward > r_max:\n",
    "            break\n",
    "        t+=1\n",
    "        if t_max > 0 and t == t_max:\n",
    "            break\n",
    "\n",
    "    \n",
    "    if return_hist_reward>=episode_reward:\n",
    "        return_val = [np.array(o_list),np.array(a_list),np.array(r_list)]\n",
    "    else:\n",
    "        return_val = episode_reward\n",
    "    return return_val\n",
    "\n",
    "# experiment collect observations and rewards for each episode\n",
    "def experiment(env, policy, n_episodes,theta=None, r_max=0, t_max=0, return_hist_reward=0):\n",
    "    \n",
    "    if return_hist_reward>0:\n",
    "        obs_list=[]\n",
    "        action_list=[]\n",
    "    else:\n",
    "        rewards=np.empty(shape=[n_episodes])\n",
    "    for i in range(n_episodes):\n",
    "        val = episode(env,policy,theta, r_max, t_max,return_hist_reward)\n",
    "        if isinstance(val, list):\n",
    "            obs_list.append(val[0])\n",
    "            action_list.append(val[1])\n",
    "            if not return_hist_reward:\n",
    "                rewards[i]=np.sum(val[2])\n",
    "        else:\n",
    "            if not return_hist_reward:\n",
    "                rewards[i]=val\n",
    "            \n",
    "    if return_hist_reward>0:\n",
    "        return_val = [np.concatenate(obs_list,axis=0), np.concatenate(action_list,axis=0)]\n",
    "    else:\n",
    "        return_val = []\n",
    "        print('Policy:{}, Min reward:{}, Max reward:{}, Average reward:{}'\n",
    "          .format(policy.__name__,\n",
    "                  np.min(rewards),\n",
    "                  np.max(rewards),\n",
    "                  np.mean(rewards)))\n",
    "\n",
    "    return return_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_7 (Dense)              (None, 8)                 40        \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 2)                 18        \n",
      "=================================================================\n",
      "Total params: 58\n",
      "Trainable params: 58\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# build the model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "model = Sequential()\n",
    "model.add(Dense(8,input_dim=4, activation='relu'))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy',optimizer='adam')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5933, 4) (5933, 2)\n"
     ]
    }
   ],
   "source": [
    "# create training data\n",
    "env = gym.make('CartPole-v0')\n",
    "n_obs = 4\n",
    "n_actions = 2\n",
    "theta = np.random.rand(4) * 2 - 1\n",
    "n_episodes = 100\n",
    "r_max = 0\n",
    "t_max = 0\n",
    "\n",
    "x_train, y_train = experiment(env, \n",
    "                              policy_random, \n",
    "                              n_episodes,\n",
    "                              theta, \n",
    "                              r_max, \n",
    "                              t_max, \n",
    "                              return_hist_reward=100 )\n",
    "y_train = np.eye(n_actions)[y_train]\n",
    "print(x_train.shape,y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "5933/5933 [==============================] - 2s 365us/step - loss: 0.5358\n",
      "Epoch 2/50\n",
      "5933/5933 [==============================] - 2s 292us/step - loss: 0.2386\n",
      "Epoch 3/50\n",
      "5933/5933 [==============================] - 2s 262us/step - loss: 0.1447\n",
      "Epoch 4/50\n",
      "5933/5933 [==============================] - 1s 245us/step - loss: 0.1129\n",
      "Epoch 5/50\n",
      "5933/5933 [==============================] - 1s 232us/step - loss: 0.0963\n",
      "Epoch 6/50\n",
      "5933/5933 [==============================] - 1s 252us/step - loss: 0.0858\n",
      "Epoch 7/50\n",
      "5933/5933 [==============================] - 2s 255us/step - loss: 0.0787\n",
      "Epoch 8/50\n",
      "5933/5933 [==============================] - 2s 294us/step - loss: 0.0728\n",
      "Epoch 9/50\n",
      "5933/5933 [==============================] - 2s 269us/step - loss: 0.0682\n",
      "Epoch 10/50\n",
      "5933/5933 [==============================] - 2s 330us/step - loss: 0.0644\n",
      "Epoch 11/50\n",
      "5933/5933 [==============================] - 2s 363us/step - loss: 0.0610\n",
      "Epoch 12/50\n",
      "5933/5933 [==============================] - 2s 337us/step - loss: 0.0586\n",
      "Epoch 13/50\n",
      "5933/5933 [==============================] - 2s 289us/step - loss: 0.0561\n",
      "Epoch 14/50\n",
      "5933/5933 [==============================] - 1s 230us/step - loss: 0.0539\n",
      "Epoch 15/50\n",
      "5933/5933 [==============================] - 1s 226us/step - loss: 0.0517\n",
      "Epoch 16/50\n",
      "5933/5933 [==============================] - 2s 267us/step - loss: 0.0499\n",
      "Epoch 17/50\n",
      "5933/5933 [==============================] - 3s 430us/step - loss: 0.0482\n",
      "Epoch 18/50\n",
      "5933/5933 [==============================] - 2s 321us/step - loss: 0.0465\n",
      "Epoch 19/50\n",
      "5933/5933 [==============================] - 2s 281us/step - loss: 0.0451\n",
      "Epoch 20/50\n",
      "5933/5933 [==============================] - 2s 321us/step - loss: 0.0439\n",
      "Epoch 21/50\n",
      "5933/5933 [==============================] - 2s 288us/step - loss: 0.0425\n",
      "Epoch 22/50\n",
      "5933/5933 [==============================] - 2s 267us/step - loss: 0.0416\n",
      "Epoch 23/50\n",
      "5933/5933 [==============================] - 2s 272us/step - loss: 0.0402\n",
      "Epoch 24/50\n",
      "5933/5933 [==============================] - 2s 316us/step - loss: 0.0393\n",
      "Epoch 25/50\n",
      "5933/5933 [==============================] - 1s 251us/step - loss: 0.0382\n",
      "Epoch 26/50\n",
      "5933/5933 [==============================] - 2s 284us/step - loss: 0.0372\n",
      "Epoch 27/50\n",
      "5933/5933 [==============================] - 2s 325us/step - loss: 0.0365\n",
      "Epoch 28/50\n",
      "5933/5933 [==============================] - 2s 256us/step - loss: 0.0355\n",
      "Epoch 29/50\n",
      "5933/5933 [==============================] - 2s 281us/step - loss: 0.0346\n",
      "Epoch 30/50\n",
      "5933/5933 [==============================] - 2s 399us/step - loss: 0.0338\n",
      "Epoch 31/50\n",
      "5933/5933 [==============================] - 2s 392us/step - loss: 0.0333\n",
      "Epoch 32/50\n",
      "5933/5933 [==============================] - 3s 440us/step - loss: 0.0325\n",
      "Epoch 33/50\n",
      "5933/5933 [==============================] - 2s 355us/step - loss: 0.0316\n",
      "Epoch 34/50\n",
      "5933/5933 [==============================] - 2s 359us/step - loss: 0.0311\n",
      "Epoch 35/50\n",
      "5933/5933 [==============================] - 2s 279us/step - loss: 0.0305\n",
      "Epoch 36/50\n",
      "5933/5933 [==============================] - 2s 418us/step - loss: 0.0297\n",
      "Epoch 37/50\n",
      "5933/5933 [==============================] - 2s 384us/step - loss: 0.0292\n",
      "Epoch 38/50\n",
      "5933/5933 [==============================] - 2s 301us/step - loss: 0.0291\n",
      "Epoch 39/50\n",
      "5933/5933 [==============================] - 2s 308us/step - loss: 0.0283\n",
      "Epoch 40/50\n",
      "5933/5933 [==============================] - 2s 273us/step - loss: 0.0279\n",
      "Epoch 41/50\n",
      "5933/5933 [==============================] - 2s 323us/step - loss: 0.0272\n",
      "Epoch 42/50\n",
      "5933/5933 [==============================] - 2s 281us/step - loss: 0.0267\n",
      "Epoch 43/50\n",
      "5933/5933 [==============================] - 2s 336us/step - loss: 0.0266\n",
      "Epoch 44/50\n",
      "5933/5933 [==============================] - 2s 392us/step - loss: 0.0260\n",
      "Epoch 45/50\n",
      "5933/5933 [==============================] - 2s 339us/step - loss: 0.0256\n",
      "Epoch 46/50\n",
      "5933/5933 [==============================] - 2s 266us/step - loss: 0.0253\n",
      "Epoch 47/50\n",
      "5933/5933 [==============================] - 2s 378us/step - loss: 0.0247\n",
      "Epoch 48/50\n",
      "5933/5933 [==============================] - 2s 343us/step - loss: 0.0248\n",
      "Epoch 49/50\n",
      "5933/5933 [==============================] - 1s 250us/step - loss: 0.0242\n",
      "Epoch 50/50\n",
      "5933/5933 [==============================] - 2s 289us/step - loss: 0.0239\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3b48175cc0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train the model\n",
    "model.fit(x_train, y_train, epochs=50, batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy:policy_naive_nn, Min reward:40.0, Max reward:107.0, Average reward:65.595\n",
      "Policy:policy_random, Min reward:41.0, Max reward:122.0, Average reward:65.175\n"
     ]
    }
   ],
   "source": [
    "n_episodes = 200\n",
    "r_max = 0\n",
    "t_max = 0\n",
    "\n",
    "_ = experiment(env, \n",
    "              policy_naive_nn, \n",
    "              n_episodes,\n",
    "              theta=model, \n",
    "              r_max=r_max, \n",
    "              t_max=t_max, \n",
    "              return_hist_reward=0 )\n",
    "\n",
    "_ = experiment(env, \n",
    "              policy_random, \n",
    "              n_episodes,\n",
    "              theta, \n",
    "              r_max, \n",
    "              t_max, \n",
    "              return_hist_reward=0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
